﻿var superagent=require('superagent')
var async=require('async')
var cheerio=require('cheerio')
var eventproxy=require('eventproxy')
var ep=new eventproxy()
var mongoose = require('mongoose')
var mongourl = 'mongodb://127.0.0.1:27017'


require('superagent-proxy')(superagent);
var proxy = process.env.http_proxy || 'http://H21XTU8514I45CDD:EE3F8E452496F88F@proxy.abuyun.com:9020';


mongoose.Promise = global.Promise; 
mongoose.connect(mongourl,{useMongoClient:true}); 
var Schema = mongoose.Schema;  
//骨架模版  
const movieSchema = new Schema({  
    text:String,
})  
//模型  
const Movie = mongoose.model('Movie', movieSchema);  
//存储数据  


var breakNum=0
breakNum=7016

var jihaiUrl='https://geohey.com/s/public_data/list/?&limit=7300&search='
var urls=[]
var erjiUrl=[]
var cookie='MEIQIA_PANEL_VISIBILITY=invisible;MEIQIA_TRACK_ID=0ssR8piCoELiEYPcgeOJ9ycshZz;captcha_key=999d1a3dcf9a45d9854708d7bf9520d3;session=eyJfaWQiOnsiIGIiOiJaVGRpTW1FMlpXSTRabVJoT1dZeE9ERXdOR05oTnpJM00yTmtaREJsWXpjPSJ9fQ.DIgMLw.bSZqlFa9wsS85jLTSYlJ1mbmgjQ;token=pFMe4AQSpLvT5U4AgJqpxtYWlZN_ybYP1SkGIlBI7cz0auTXC5gmQcP1Y07EpQ7C'
superagent.get(jihaiUrl)
    .proxy(proxy)
    .set("Cookie",cookie)
    .end(function(err,res){
        if(err)
            console.error('错误')
        else{
            isLogin(res.text)
        }
    })
    function isLogin(text){
        var data=JSON.parse(text)
        for(var i=breakNum;i<data.data.items.length;i++){
            if(data.data.items[i].uid=='administration_nature_place_name'){
                console.log(i,data.data.items[i].count)
                if(data.data.items[i].count>500){
                    for(var j=414500;j<Math.ceil(data.data.items[i].count/500)*500;j++){
                        if(j%500==0){
                            urls.push({url:'https://geohey.com/s/public_data/'+data.data.items[i].uid+'/query?limit=500&offset='+j,name:data.data.items[i].name})
                        }
                    }
                }else{
                    urls.push({url:'https://geohey.com/s/public_data/'+data.data.items[i].uid+'/query?limit=500&offset=',name:data.data.items[i].name})
                }
            }else{
                // if(data.data.items[i].count>500){
                //     for(var j=0;j<Math.ceil(data.data.items[i].count/500)*500;j++){
                //         if(j%500==0){
                //             urls.push({url:'https://geohey.com/s/public_data/'+data.data.items[i].uid+'/query?limit=500&offset='+j,name:data.data.items[i].name})
                //         }
                //     }
                // }else{
                //     urls.push({url:'https://geohey.com/s/public_data/'+data.data.items[i].uid+'/query?limit=500&offset=',name:data.data.items[i].name})
                // }
            }
            // if(data.data.items[i].count>500){
            //     for(var j=0;j<Math.ceil(data.data.items[i].count/500)*500;j++){
            //         if(j%500==0){
            //             urls.push({url:'https://geohey.com/s/public_data/'+data.data.items[i].uid+'/query?limit=500&offset='+j,name:data.data.items[i].name})
            //         }
            //     }
            // }else{
            //     urls.push({url:'https://geohey.com/s/public_data/'+data.data.items[i].uid+'/query?limit=500&offset=',name:data.data.items[i].name})
            // }
        }
        getData()
    }
    function getData(){
        var concurrencyCount = 0
        var itemArr = []
        var fetchUrl = function(url, callback){
            // var delay = parseInt((Math.random() * 10000000) % 2000, 10)
            var delay = 5000
            concurrencyCount ++
            console.log('并发数：'+ concurrencyCount,'正在抓取：',url.name,url.url,'耗时：（ms）'+delay)
            superagent.get(url.url)
                .proxy(proxy)
                .set("Cookie",cookie)
                .end(function(err,res){
                    if(res){
                        var text=JSON.parse(res.text)
                        if(text){
                            if(!text.data.features=='undefined'){
                                text.data.features.forEach(function(item){
                                    const Movie = mongoose.model(url.name, movieSchema);  
                                    const moive = new Movie({
                                        text:JSON.stringify(item),
                                    })  
                                    //保存数据库  
                                    moive.save(function(err) {
                                        if (err) {  
                                            console.log('保存失败')  
                                            return;  
                                        }              
                                    }) 
                                })
                            }
                        }
                    }else{
                        console.log(url+"错误")
                    }
                })
            setTimeout(function(){
                concurrencyCount --
                callback(null,url.url + 'html content')
            },delay)
        }

        async.mapLimit(urls,15,
            function(url, callback){
                fetchUrl(url, callback)
            },
            function(err, result){
                console.log('final:')
                console.log(result)
                console.log(itemArr)
            })
    }













